{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第二步：调整树的参数：max_depth & min_child_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from matplotlib import pyplot\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "import seaborn as sns\n",
    "\n",
    "from numpy import nan as NaN\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('FE_X_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Disbursed属性的不同取值和出现的次数\n",
      "0.0    4694\n",
      "1.0     264\n",
      "Name: Disbursed, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#对类别型特征，观察其取值范围及直方图\n",
    "categorical_features = ['Disbursed']\n",
    "for col in categorical_features:\n",
    "    print('\\n%s属性的不同取值和出现的次数'%col)\n",
    "    print(train[col].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['Disbursed']\n",
    "\n",
    "train = train.drop([\"Disbursed\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [1, 2, 3], 'min_child_weight': [0.01, 0.5, 1]}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##### max_depth 3-10， min_child_weight 1-6\n",
    "# max_depth = range(3,10,2)\n",
    "# min_child_weight = range(1,6,2)\n",
    "max_depth = [1,2,3]\n",
    "min_child_weight = [0.01,0.5,1]\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(make_scorer(log_loss, greater_is_better=False, needs_proba=True),\n",
       " {'max_depth': 2, 'min_child_weight': 0.01},\n",
       " -0.251843451314537)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)\n",
    "\n",
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=14,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'binary:logistic',\n",
    "#         num_class = 9,\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=2, cv=kfold)\n",
    "gsearch2_1.fit(X_train, y_train)\n",
    "\n",
    "gsearch2_1.scorer_, gsearch2_1.best_params_, gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.04788518, 0.05176158, 0.04831023, 0.06892667, 0.0600738 ,\n",
       "        0.05703211, 0.07189326, 0.07269807, 0.07012095]),\n",
       " 'std_fit_time': array([0.00155769, 0.00501708, 0.00543177, 0.01200106, 0.0037389 ,\n",
       "        0.00105797, 0.00185334, 0.00094995, 0.00078824]),\n",
       " 'mean_score_time': array([0.00291648, 0.00341477, 0.00268302, 0.00340528, 0.00258651,\n",
       "        0.00258112, 0.00260329, 0.00262218, 0.00264821]),\n",
       " 'std_score_time': array([1.59971909e-04, 1.56018291e-04, 2.49205181e-04, 7.65604333e-04,\n",
       "        1.64392306e-04, 1.95276315e-05, 1.22211140e-04, 4.78881744e-05,\n",
       "        3.43784710e-05]),\n",
       " 'param_max_depth': masked_array(data=[1, 1, 1, 2, 2, 2, 3, 3, 3],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[0.01, 0.5, 1, 0.01, 0.5, 1, 0.01, 0.5, 1],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 1, 'min_child_weight': 0.01},\n",
       "  {'max_depth': 1, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 1, 'min_child_weight': 1},\n",
       "  {'max_depth': 2, 'min_child_weight': 0.01},\n",
       "  {'max_depth': 2, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 2, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 0.01},\n",
       "  {'max_depth': 3, 'min_child_weight': 0.5},\n",
       "  {'max_depth': 3, 'min_child_weight': 1}],\n",
       " 'split0_test_score': array([-0.25136182, -0.25136182, -0.25136182, -0.24886082, -0.24897868,\n",
       "        -0.24947315, -0.24892153, -0.24964766, -0.24986181]),\n",
       " 'split1_test_score': array([-0.25395572, -0.25395572, -0.25395572, -0.24947394, -0.24946856,\n",
       "        -0.24943061, -0.24920154, -0.24914915, -0.24882816]),\n",
       " 'split2_test_score': array([-0.25423098, -0.25423098, -0.25423098, -0.25366797, -0.25346695,\n",
       "        -0.25372627, -0.25421304, -0.25437979, -0.2545173 ]),\n",
       " 'split3_test_score': array([-0.25670211, -0.25670211, -0.25670211, -0.25524608, -0.25547737,\n",
       "        -0.25534562, -0.25497486, -0.25489321, -0.25456311]),\n",
       " 'split4_test_score': array([-0.25601938, -0.25601938, -0.25601938, -0.25196869, -0.25196869,\n",
       "        -0.25201236, -0.25278029, -0.25251022, -0.25284996]),\n",
       " 'mean_test_score': array([-0.25445337, -0.25445337, -0.25445337, -0.25184345, -0.25187201,\n",
       "        -0.2519976 , -0.25201794, -0.25211585, -0.25212377]),\n",
       " 'std_test_score': array([0.00186366, 0.00186366, 0.00186366, 0.00242672, 0.00243769,\n",
       "        0.00233114, 0.00251692, 0.00236215, 0.00237464]),\n",
       " 'rank_test_score': array([7, 7, 7, 1, 2, 3, 4, 5, 6], dtype=int32),\n",
       " 'split0_train_score': array([-0.25394534, -0.25394534, -0.25394534, -0.25126887, -0.25135365,\n",
       "        -0.25113148, -0.24832647, -0.24898581, -0.24909935]),\n",
       " 'split1_train_score': array([-0.25419667, -0.25419667, -0.25419667, -0.25056432, -0.25062351,\n",
       "        -0.25088468, -0.24771745, -0.2479733 , -0.2478653 ]),\n",
       " 'split2_train_score': array([-0.25341487, -0.25341487, -0.25341487, -0.25015567, -0.2500976 ,\n",
       "        -0.25017047, -0.24643221, -0.24665392, -0.24685397]),\n",
       " 'split3_train_score': array([-0.25222009, -0.25222009, -0.25222009, -0.24836111, -0.24841724,\n",
       "        -0.24840897, -0.24447224, -0.24465   , -0.24445291]),\n",
       " 'split4_train_score': array([-0.25406395, -0.25406395, -0.25406395, -0.24915502, -0.24915502,\n",
       "        -0.24917955, -0.24682133, -0.24697964, -0.24747405]),\n",
       " 'mean_train_score': array([-0.25356818, -0.25356818, -0.25356818, -0.249901  , -0.2499294 ,\n",
       "        -0.24995503, -0.24675394, -0.24704853, -0.24714912]),\n",
       " 'std_train_score': array([0.00072442, 0.00072442, 0.00072442, 0.00102992, 0.00104162,\n",
       "        0.00102824, 0.00132026, 0.00145063, 0.00153483])}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_\n",
    "# gsearch2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # summarize results\n",
    "# print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "# test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "# test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "# train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "# train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# # plot results\n",
    "# test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "# train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "# for i, value in enumerate(max_depth):\n",
    "#     pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "# #for i, value in enumerate(min_child_weight):\n",
    "# #    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "# pyplot.legend()\n",
    "# pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "# pyplot.ylabel( 'Log Loss' )\n",
    "# pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.251843451314537"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': 2, 'min_child_weight': 0.01}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.best_params_"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
